# encoding=utf-8
import requests
from bs4 import BeautifulSoup
import os
from multiprocessing import Pool
def getConnection(url):
    html = requests.get(url)
    print(html.encoding)
    if 'ISO-8859-1' == html.encoding:
        demo = html.text.encode("ISO-8859-1").decode('gbk').encode('utf8')
    else:
        demo = html.text
    soup = BeautifulSoup(demo, 'html.parser')
    return soup
def getUrl(tagName, soup):
    tag = soup.find_all(tagName, target ='_blank')
    return tag

def getImage(tagName,soup):
    tag = soup.find_all(tagName)
    root = "D://wallpaper//"
    for i in range(0,len(tag)):
        url=tag[i].attrs['src']
        path = root + str(tag[i].get('alt'))+".jpg"
        print(url)
        print(path)
        try:
            if not os.path.exists(root):
                os.mkdir(root)
            #使用with语句可以不用自己手动关闭已经打开的文件流
            r = requests.get(url)
            r.raise_for_status()
            with open(path, "wb") as f: #开始写文件，wb代表写二进制文件
                f.write(r.content)
            print("第"+str(i)+"次爬取完成")
        except Exception as e:
            print("第"+str(i)+"次爬取失败"+"/r"+str(e))
def main():
    for i in range(0,5000):
        if i == 0:
            url = 'http://www.netbian.com/s/chaogaoqing/'
        else:
            url = 'http://www.netbian.com/meinv/index_' + str(i) + '.htm'
        soup = getConnection(url)
        tag = getUrl('a',soup)
        print(url)
        for i in range(0, len(tag)):
            try:
                url = 'http://www.netbian.com' + tag[i].get('href')
                if 'desk' not in url:
                    continue
                soup = getConnection(url)
                tagI = soup.find('div', class_='pic-down')
                href = tagI.a.get('href')
                urlP = 'http://www.netbian.com' + href
                soupI = getConnection(urlP)
                getImage('img', soupI)
            except Exception as e:
                print('获取地址异常' + str(e))
if __name__ == '__main__':
    #创建10个线程
    pool = Pool(10)
    pool.map(main())

